Web Crawler Practice :: Dynamic Webpage (TikTok)¶

Dr. Chan, Chun-Hsiang
Department of Geography, National Taiwan Normal University

In [52]:
# import packages
import os
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.common.keys import Keys
from bs4 import BeautifulSoup
In [14]:
# set parameters
url = 'https://www.tiktok.com/explore?lang=zh-Hant-TW'
In [55]:
# start a webdriver
browser = webdriver.Chrome()
# enter url
browser.get(url)
In [56]:
# enter the webpage as a guest
button_xPath = '/html/body/div[5]/div[3]/div/div/div/div[1]/div/div/div[3]/div/div[2]/div'
guest_button = browser.find_element(By.XPATH, button_xPath)
guest_button.click()
In [57]:
# find element
content_xPath = '/html/body/div[1]/div[2]/div[2]/div/div[2]/div'
content_ = browser.find_element(By.XPATH, content_xPath)
In [58]:
# show content
content_.text
Out[58]:
"1.7M\nacara hr kartini malah mberot😅😅\nlungitingasmoro1\n66.3K\n2M\nDrama nok arep metu🤭mesti leren ngereog sek ma ama🥺 kerjo dolan bayaran👍 syahdunee\U0001faf6\nuwasin\n66.5K\n1.2M\n日進和尚來台super house#dj #和尚 #嘻哈 #夜店 #有趣\nminjun_182\n41.7K\n1.7M\n#assalamualaikum #fyp #fypシ #tetepsemangat #tiktokhiburan\nputryragil21\n100K\n1.4M\nSPOT! #SPOT #spotchallenge #JENNIE #ZICO #dance #kpop\n_5277_\n159.9K\n1.2M\nLet's say BOOYAH! #你最愛的大逃殺手遊 #FREEFIRE @imshowmain\nfreefire_tw\n6961\n864.1K\n在南鯤鯓巧遇潮六\nyoyo_1009\n51.1K\n784.2K\n回覆 @云淡风轻\naa0983975hp\n18.7K\n5.8M\n「Magnetic 🤟🏻」流行舞蹈教學🔍#fyp #magnetic #dance #illit\nevanjoe0106\n433.3K\n1M\n這種風格不是我的強項 旁邊的比較會的感覺#06 #fypシ\n_kong_824\n69.2K\n1.6M\n🤣\nadlysdyhayu_87\n59K\n7.1M\nmy fyp rn for 24/7 // ins: @itsmichellekwok #fyp\nitsmichellekwok\n300.1K\n995.5K\nBụi bặm phong cách khác lạ\nMình thích cách bạn ấy hát giọng rất đặc biệt.\nadamtuan86\n28.1K\n860.3K\n斗音发现的神翻唱！斗音@权志驴\n完整版在@恭喜发财\n\n\n\nalex3334133341\n22K"
In [59]:
# select item by class containing specific words
item_className = 'DivItemContainerV2'
item_list = content_.find_element(By.CSS_SELECTOR, "div[class*='"+item_className+"']")
In [60]:
# select item by class containing specific words
video_className = 'DivWrapper'
video_viewer = item_list.find_elements(By.CSS_SELECTOR, "div[class*='"+video_className+"']")[0]
video_viewer.text
Out[60]:
'1.7M'
In [61]:
# select item by class containing specific words and send keys
search_textbox = browser.find_element(By.CSS_SELECTOR, "input[placeholder*='搜尋']")
search_textbox.send_keys('臺灣')
In [62]:
# select item by class containing specific words and click
search_button = browser.find_element(By.CSS_SELECTOR, "button[data-e2e*='search-box-button']")
search_button.click()
In [63]:
# select item by class containing specific words and clear
search_textbox = browser.find_element(By.CSS_SELECTOR, "input[placeholder*='搜尋']")
search_textbox.clear()
In [64]:
# transform source code to bs4 soup
soup = BeautifulSoup(browser.page_source)
In [67]:
# print text
soup.text
Out[67]:
"在 TikTok 上搜尋'臺灣' | TikTok 搜尋  TikTok臺灣上傳 登入為您推薦關注中好友探索直播個人資料登入可關注創作者、對影片按讚以及查看評論。 登入創作 TikTok 特效，獲得獎勵公司關於新聞編輯室聯絡方式工作計劃TikTok for Good廣告TikTok LIVE Creator NetworksDevelopers透明度TikTok 獎勵TikTok Embeds條款與政策說明安全條款隱私權政策隱私權中心Creator Academy社群自律公約© 2024 TikTok熱門帳號影片直播2023-9-20你以為的台灣#台灣  #中華民國  #🇹🇼  #👍  #taiwan   roc93578002.9M2023-1-29這些台灣話你可能聽不懂#漲知識  \xa0#財經  \xa0#語言  \xa0#台灣腔  dongtaizai52.9K2023-10-11大陸的朋友逛寧夏夜市，吃吃吃 #大陸人遊台灣  #搬運  #大陸人眼中的台灣  #夜市  #寧夏夜市  #台灣  #美食   resetchentaoyuan235.9K2023-11-21台灣妹妹，很可愛，喜歡就關注哦  #Flala  flalalove1.8M3-20全系列三隻羊產品代購！ 你想的到的全部都買得到！ 趕緊點我主頁連結購買！ #三隻羊代購  #上推薦  #台灣  #瘋狂小楊哥  #小楊臻選  #正品代購   crazy_yang_buy902K1週前 住台灣能有多爽？ #空姐姐  #世界排行  #旅遊  #居住  #世界第一  #知識  #經濟  #台灣   beatrice77889928.4K4-3地震后亲眼目睹山崩，台湾女游客崩溃大哭“我不要玩啦，都是你啦”，男子懵了：我怎样啊？ joyihlnukys984K2022-9-19一名郭姓女网友台湾地震发生时，刚好在全台最高的101大楼89楼，目睹大楼内的阻尼器剧烈狂晃，画面曝光后引发热议！#光华日报  #光华报动  #BeritaDiTiktok  #NewsAtTiktok  #TikTok新闻  #TrendingNewsMalaysia  kwongwah1910136.2K2-3（台大街訪）台灣是中國的一部分嗎？#柳傑克  #台灣  #兩岸   iamjackliou493K3-1台灣地圖上顯示台灣省#台灣  #台灣省  #台北  beizhenfusi108.5K1-30#台灣當歸  #祖國統一  s543838227.7K5天前 #花蓮外海  #驚奇  #台灣地震   bb73925372.5K取得應用程式取得 TikTok 桌面版取得 TikTok 應用程式"
In [79]:
# get video information
video_info = soup.find_all("div", {'class':'css-1soki6-DivItemContainerForSearch e19c29qe10'})
video_info_ = video_info[0]
video_info_
Out[79]:
<div class="css-1soki6-DivItemContainerForSearch e19c29qe10"><div aria-label="以全螢幕觀看" class="css-bbkab3-DivContainer e1cg0wnj0" data-e2e="search_top-item" role="button" tabindex="0"><div style="padding-top: 133.188%;"><div class="css-1as5cen-DivWrapper e1cg0wnj1"><a href="https://www.tiktok.com/@roc9357800/video/7280847625014742277" tabindex="-1"><canvas class="css-1yvkaiq-CanvasPlaceholder e19c29qe2" height="100" width="75.08196721311475"></canvas><div class="css-1wa52dp-DivPlayerContainer e19c29qe5"><div class="css-1jxhpnd-DivContainer e1yey0rl0" mode="1"><div class="css-41hm0z"><span style="box-sizing: border-box; display: block; overflow: hidden; width: initial; height: initial; background: none; opacity: 1; border: 0px; margin: 0px; padding: 0px; position: absolute; inset: 0px;"><picture><img alt="🇹🇼❤️ 使用 🇹🇼❤️ 的 原聲 - 🇹🇼🤔 創作的 你以為的台灣#台灣 #中華民國 #🇹🇼 #👍 #taiwan " decoding="async" imagex-type="react" imagex-version="0.3.10" src="https://p16-sign-va.tiktokcdn.com/obj/tos-maliva-p-0068/9dea0c5f878d424cafbd882c18c69ddf_1695204447?x-expires=1715090400&amp;x-signature=tUKowMzBkabjXBj%2F6%2FjeHaMH7xc%3D" srcset="https://p16-sign-va.tiktokcdn.com/obj/tos-maliva-p-0068/9dea0c5f878d424cafbd882c18c69ddf_1695204447?x-expires=1715090400&amp;x-signature=tUKowMzBkabjXBj%2F6%2FjeHaMH7xc%3D 1x, https://p16-sign-va.tiktokcdn.com/obj/tos-maliva-p-0068/9dea0c5f878d424cafbd882c18c69ddf_1695204447?x-expires=1715090400&amp;x-signature=tUKowMzBkabjXBj%2F6%2FjeHaMH7xc%3D 2x" style="position: absolute; inset: 0px; box-sizing: border-box; padding: 0px; border: none; margin: auto; display: block; width: 0px; height: 0px; min-width: 100%; max-width: 100%; min-height: 100%; max-height: 100%;"/></picture></span></div><div class="css-1fofj7p-DivBasicPlayerWrapper e1yey0rl2"><div class="tiktok-web-player no-controls" id="xgwrapper-1-7280847625014742277" style="width: 100%; height: 100%;"><video crossorigin="use-credentials" playsinline="" preload="auto" src="https://v16-webapp-prime.tiktok.com/video/tos/useast2a/tos-useast2a-pve-0068/oIIvxW2nzEgGDKMHAcfQk6IvAANUyGgIh2oxCh/?a=1988&amp;bti=NDU3ZjAwOg%3D%3D&amp;ch=0&amp;cr=3&amp;dr=0&amp;lr=unwatermarked&amp;cd=0%7C0%7C0%7C&amp;cv=1&amp;br=3204&amp;bt=1602&amp;cs=0&amp;ds=6&amp;ft=Fx9KL6BMyq8~nPnkE125L5IYztGxRf&amp;mime_type=video_mp4&amp;qs=0&amp;rc=NzQ6aTY8NTs6aGZoOWdnaUBpajN3dmY6ZnFmbjMzNzczM0AtLTYzMDQwNWMxXjIwYC4zYSMubGFncjRnNDJgLS1kMTZzcw%3D%3D&amp;btag=e00088000&amp;expire=1715092473&amp;l=202405051434010E24D77BD079249C561B&amp;ply_type=2&amp;policy=2&amp;signature=5d89443d9135ab0405851eded174fc91&amp;tk=tt_chain_token" style="width: 100%; height: 100%;"></video></div></div></div><div class="css-11u47i-DivCardFooter e148ts220"><div class="css-dennn6-DivTimeTag e19c29qe15">2023-9-20</div></div></div></a></div></div></div><div class="css-hbrxqe-DivVideoSearchCardDesc etrd4pu0" data-e2e="search-card-desc"><div class="css-9z9m4b-DivCardBottomInfo etrd4pu2"><div class="css-f22ew5-DivMetaCaptionLine etrd4pu1" data-e2e="search-card-video-caption"><div class="css-1iy6zew-DivContainer ejg0rhn0"><span class="css-j2a19r-SpanText efbd9f0">你以為的台灣</span><a aria-label="Watch more videos of the #台灣 category" class="ejg0rhn6 css-g8ml1x-StyledLink-StyledCommonLink er1vbsz0" data-e2e="search-common-link" href="/tag/台灣?lang=zh-Hant-TW" rel="opener" target="_self"><strong class="css-1p6dp51-StrongText ejg0rhn2" color="rgba(43, 93, 185, 1)">#台灣 </strong></a><span class="css-j2a19r-SpanText efbd9f0"> </span><a aria-label="Watch more videos of the #中華民國 category" class="ejg0rhn6 css-g8ml1x-StyledLink-StyledCommonLink er1vbsz0" data-e2e="search-common-link" href="/tag/中華民國?lang=zh-Hant-TW" rel="opener" target="_self"><strong class="css-1p6dp51-StrongText ejg0rhn2" color="rgba(43, 93, 185, 1)">#中華民國 </strong></a><span class="css-j2a19r-SpanText efbd9f0"> </span><a aria-label="Watch more videos of the #🇹🇼 category" class="ejg0rhn6 css-g8ml1x-StyledLink-StyledCommonLink er1vbsz0" data-e2e="search-common-link" href="/tag/🇹🇼?lang=zh-Hant-TW" rel="opener" target="_self"><strong class="css-1p6dp51-StrongText ejg0rhn2" color="rgba(43, 93, 185, 1)">#🇹🇼 </strong></a><span class="css-j2a19r-SpanText efbd9f0"> </span><a aria-label="Watch more videos of the #👍 category" class="ejg0rhn6 css-g8ml1x-StyledLink-StyledCommonLink er1vbsz0" data-e2e="search-common-link" href="/tag/👍?lang=zh-Hant-TW" rel="opener" target="_self"><strong class="css-1p6dp51-StrongText ejg0rhn2" color="rgba(43, 93, 185, 1)">#👍 </strong></a><span class="css-j2a19r-SpanText efbd9f0"> </span><a aria-label="Watch more videos of the #taiwan category" class="ejg0rhn6 css-g8ml1x-StyledLink-StyledCommonLink er1vbsz0" data-e2e="search-common-link" href="/tag/taiwan?lang=zh-Hant-TW" rel="opener" target="_self"><strong class="css-1p6dp51-StrongText ejg0rhn2" color="rgba(43, 93, 185, 1)">#taiwan </strong></a><span class="css-j2a19r-SpanText efbd9f0"> </span></div></div><div class="css-1kw4mmh-DivPlayLine etrd4pu3"><a aria-label="🇹🇼❤️ 的個人資料" class="css-22xkqc-StyledLink er1vbsz0" data-e2e="search-card-user-link" href="/@roc9357800?lang=zh-Hant-TW" id="search_top-item-user-link-0"><div class="css-dq7zy8-DivUserInfo etrd4pu5"><span class="css-tuohvl-SpanAvatarContainer e1e9er4e0" data-e2e="" shape="circle" style="flex-shrink: 0; width: 24px; height: 24px;"><img alt="" class="css-1zpj2q-ImgAvatar e1e9er4e1" loading="lazy" src="https://p16-sign-va.tiktokcdn.com/tos-maliva-avt-0068/1af44b0a14bf2a093c505920fa9e0ca0~c5_100x100.jpeg?lk3s=a5d48078&amp;x-expires=1715090400&amp;x-signature=HYUeeyHwIFPqviXaiSpd0aoBT6o%3D"/></span> <p class="css-2zn17v-PUniqueId etrd4pu6" data-e2e="search-card-user-unique-id">roc9357800</p></div></a><div class="css-1lbowdj-DivPlayIcon etrd4pu4" data-e2e="search-card-like-container"><svg class="like-icon css-b82ygf-StyledPlay etrd4pu9" data-e2e="" fill="rgba(22, 24, 35, 0.75)" height="16" viewbox="0 0 48 48" width="16" xmlns="http://www.w3.org/2000/svg"><path clip-rule="evenodd" d="M16 10.554V37.4459L38.1463 24L16 10.554ZM12 8.77702C12 6.43812 14.5577 4.99881 16.5569 6.21266L41.6301 21.4356C43.5542 22.6038 43.5542 25.3962 41.6301 26.5644L16.5569 41.7873C14.5577 43.0012 12 41.5619 12 39.223V8.77702Z" fill-rule="evenodd"></path></svg><strong class="css-ws4x78-StrongVideoCount etrd4pu10">2.9M</strong></div></div></div></div></div>
In [93]:
# get url
video_link = video_info_.find_all('div', {'class':'css-1as5cen-DivWrapper e1cg0wnj1'})[0]
video_link.find_all('a')[0]['href']
Out[93]:
'https://www.tiktok.com/@roc9357800/video/7280847625014742277'
In [95]:
# get date info
video_info_.find_all('div', {'class':'css-11u47i-DivCardFooter e148ts220'})[0].text
Out[95]:
'2023-9-20'
In [96]:
# get title
video_info_.find_all('div', {'class':'css-1iy6zew-DivContainer ejg0rhn0'})[0].text
Out[96]:
'你以為的台灣#台灣  #中華民國  #🇹🇼  #👍  #taiwan  '
In [99]:
# get user id
video_info_.find_all('a', {'data-e2e':'search-card-user-link'})[0].text
Out[99]:
' roc9357800'
In [100]:
# get user link
video_info_.find_all('a', {'data-e2e':'search-card-user-link'})[0]['href']
Out[100]:
'/@roc9357800?lang=zh-Hant-TW'
In [102]:
# get view number
video_info_.find_all('strong', {'class':'css-ws4x78-StrongVideoCount etrd4pu10'})[0].text
Out[102]:
'2.9M'
In [103]:
# close browser
browser.quit()
In [ ]: